// curl -k https://localhost:8080/
const https = require('https')
const cheerio = require('cheerio')
const targetUrl = 'https://news.sina.com.cn/china/'
const fs = require('fs')
const axios = require('axios')
var html = ''



//只用做发请求
function request(url){
    var html = ''
    var pro = new Promise((resolve)=>{
        https.get(url,(response)=>{
            response.setEncoding('utf-8')
            response.on('data',(chunk)=>{
                html = html.concat(chunk)
            })
            response.on('end',()=>{
                //console.log("没有数据了")
                resolve(html)
            })
        })
    }) 
    return pro
}

//只用做解析html并直接写进文档
function copeWithHtml(url,isDetail){
    var req = request(url)
    req.then((res)=>{
        if(!isDetail){
            //如果不是详情页，即为爬取首页，则执行以下
            let $ = cheerio.load(res, {decodeEntities: false})
            let targetUl = null
            let a = null
            //遍历div为标签的
            $('div').map((index,element)=>{
                //找到没有class的div,并赋值
                if(!element.attribs.class){
                    targetUl = element.children[1]
                }
            })
            //获取目标Ul元素的子元素【a标签】
            a = $(targetUl).find('a')
            //遍历a标签
            for(let i=0; i<a.length; i++){
                //存在title
                if(a[i].attribs.title){
                    //获取到所有a链接及内容
                    //console.log(a[i].attribs.href)
                    //二次此请求
                    copeWithHtml(a[i].attribs.href,true)
                }
            }
        }else{
            //若是详情页
            //console.log($('.article').find('p').text())
            //console.log($('.img_wrapper').find('img')[0].attribs.src)
            let $ = cheerio.load(res , {decodeEntities: false})
            let restContent = `${$('.main-title').text()}\r\n${$('.date').text()}\r\n${$('.article').find('p').text()}\r\n`
            $('.img_wrapper').find('img').map((index,element)=>{
                restContent = restContent.concat(element.attribs.src)
                restContent += `\r\n`
            })
            fs.appendFile('./新闻详情.txt',restContent,(err)=>{
                if(err){
                    console.log(err)
                    throw err
                }
            })
        }
    })
}

copeWithHtml(targetUrl,false)


































// https.get(targetUrl,(response)=>{
//     //设置相应格式
//     response.setEncoding('utf-8')
//     //监听数据获取的过程
//     response.on('data',(chunk)=>{
//         html = html.concat(chunk)
//     })
//     //监听数据获取完成
//     response.on('end',()=>{
//         console.log("没有数据了")
//         let $ = cheerio.load(html, {decodeEntities: false})
//         let targetUl = null
//         let a = null
//         //遍历div为标签的
//         $('div').map((index,element)=>{
//             //找到没有class的div,并赋值
//             if(!element.attribs.class){
//                 targetUl = element.children[1]
//             }
//         })
//         //获取目标Ul元素的子元素【a标签】
//         a = $(targetUl).find('a')
//         //遍历a标签
//         for(let i=0; i<a.length; i++){
//             //存在title
//             if(a[i].attribs.title){
//                 //获取到所有a链接及内容
//                 console.log(a[i].attribs)
//             }
//         }
//     })
// })
































// function copeWithHtml(html){
//         html = html.split("<!--feed流配置-->")[1]
//         html = html.split("<!--/feed流配置-->")[0]
//         html = html.split('</script>')[0]
//         html = html.split("tabInfos")[1]
//         html = html.split("requireLogin")[0]
//         html = html.split("/*")[1]
//         html = html.split("*/")[0]
//         html = html.split("'")[5]
//         html = html.replace("http","https")
//         html = html.replace("mix.","")
//         html = html.replace("155","121")
//         html = html.replace("1686","1356")
//         html = html.replace("10","20")
//         return html
// }

/*function httpsRequest(){
    const request = https.get(targetUrl,(result)=>{
        result.setEncoding('utf-8')
        result.on('data',(chunk)=>{
            html = html.concat(chunk)
        })
        result.on('end',()=>{
            console.log('没有数据了')
            html = copeWithHtml(html)
            axios({
                url:html,
                method:'get'
            }).then((res)=>{
                console.log(res.data.result.data)
            })
        })
    })
    request.on('error',(e)=>{
        console.log(`error:${e}`)
    })
}

httpsRequest();
*/
